import requests
from bs4 import BeautifulSoup
from openpyxl import Workbook
from openpyxl.compat import range
import datetime
import time
import gc


class ProvCityData:
    def __init__(self):
        self.province = ''
        self.cityinfo = {}

    def AddCityInfoItem(self, city, website):
        self.cityinfo[city] = website

    def printCityInfo(self):
        print(self.cityinfo)


class StationDataInfo():
    def __init__(self):
        self.stName = ''
        self.StWebSite = ''
        self.cityName = ''
        self.province = ''


class StationDatatraper():
    def __init__(self):
        self.StationDataInfoList = []
        self.url = 'http://www.pm25.com'
        self.prefix = 'http://www.pm25.com/city'
        self.ProvCityDataLists = []

    def _getStationSiteDict(self, cityWebSite):
        try:
            siteDict = {}
            # time.sleep(1)
            resp = requests.get(cityWebSite, timeout=10)
            html = resp.text
            lj = BeautifulSoup(html, 'html.parser')
            Station_list = lj.find_all('li', attrs={'class': 'pj_area_data_item pj_area_data_item_darkbg'})
            # print(Station_lists_str)
            for l in Station_list:
                # a是tag ,可以直接获取tag的属性和tag的NavigableString
                if ((not str(l.a['title']).__contains__('西溪')) or str(l.a['href']).__contains__('西溪')):
                    siteDict[l.a.string] = l.a['href']
            return siteDict
        except:
            print('get %s error' % cityWebSite)

    def _getTotalCityList(self):

        try:
            resp = requests.get(self.url, timeout=10)
            print(resp.status_code)
            html = resp.text
            lj = BeautifulSoup(html, 'html.parser')
            city_lists_str = lj.find_all('div', attrs={'class': 'city_province_item'})
            return city_lists_str


        except:
            print('error')

    def _GetProvCityDataList(self, html_str):
        for l in html_str:
            pcd = ProvCityData()
            pcd.province = l.dt.string
            if (pcd.province.__len__() >= 2):  # 过滤掉省份是字母简称的情况
                cityWebList = l.dd
                for c in cityWebList:
                    if (c != '\n'):
                        s = str(c).split('"')
                        key = (s[2])[1:-4]
                        # if key not in pcd.cityinfo:  #dist 去重
                        pcd.AddCityInfoItem(key, self.prefix + s[1])
                self.ProvCityDataLists.append(pcd)

    def _IsNotInList(self, listname, item):
        for val in listname:
            if ((val.stName == item.stName) and (val.cityName == item.cityName)):
                return False
        return True

    def _GetStationDataInfoList(self):
        self.StationDataInfoList.clear()
        for pcdItem in self.ProvCityDataLists:
            for k in pcdItem.cityinfo:
                print("%s" % k, pcdItem.cityinfo[k])
                webdict = self._getStationSiteDict(str(pcdItem.cityinfo[k]))
                if webdict is not None:
                    for d in webdict:
                        sd = StationDataInfo()
                        sd.cityName = k
                        sd.province = pcdItem.province
                        if sd.province == '直辖市':
                            sd.province = sd.cityName
                        elif sd.province == '港澳台':
                            continue
                        sd.stName = d
                        sd.StWebSite = self.changeStrToUTF8(self.url + webdict[d])[2:-1]
                        if self._IsNotInList(self.StationDataInfoList, sd):
                            self.StationDataInfoList.append(sd)
                del webdict
                gc.collect()
                # print(self.StationDataInfoList.__len__())

    def changeStrToUTF8(self, strs):
        s = str(strs.encode('utf-8'))
        return s.replace('\\x', '%')

    def _WriteToXlsx(self):

        wb = Workbook()

        dest_filename = '完整清单' + str(datetime.date.today()) + '.xlsx'

        ws1 = wb.active

        ws1.title = "sheet1"

        ws1['A1'] = '站点名称'

        ws1['B1'] = 'URL地址'

        ws1['C1'] = '城市'

        ws1['D1'] = '省份'

        ws1['E1'] = '读取标记'
        # self.StationDataInfoList = list(map(self.StationDataInfoList))
        for s in self.StationDataInfoList:
            ws1.append([s.stName, s.StWebSite, s.cityName, s.province, 1])
        wb.save(filename=dest_filename)

    def StartWork(self):
        tclist = self._getTotalCityList()
        self._GetProvCityDataList(tclist)
        self._GetStationDataInfoList()
        self._WriteToXlsx()


if __name__ == '__main__':
    g = StationDatatraper()
    g.StartWork()
